﻿<%@ Page Language="C#" %>

<script runat="server">   
    
    string collectKey = string.Empty;
    string nodeKey = string.Empty;
    int rIndex = 0;
    static int totalCount = 0;
    int collectCount = 0;
    string stautsMsg = string.Empty;
    protected override void OnLoad(EventArgs e)
    {
        Response.Buffer = true;
        Response.ExpiresAbsolute = System.DateTime.Now.AddSeconds(-1);
        Response.Expires = 0; Response.CacheControl = "no-cache";
        //
        base.OnLoad(e);
        rIndex = Request.GetQueryValue<int>("rIndex");
        collectKey = Request.GetQueryValue<string>("collectKey");
        nodeKey = Request.GetQueryValue<string>("nodeKey");
        collectCount = Request.GetQueryValue<int>("cIndex");
        Response.Write(GetResult());
    }

    public string GetResult()
    {
        //获取采集配置信息
        if (string.IsNullOrEmpty(collectKey))
            return ObjectToJson(null, 0, "ERRE:没有找到此站点的配置配置！");
        Ue.Qcdz.Domain.Collectors.Collector collModel =
            Ue.Qcdz.Domain.Collectors.CollectorService.GetCollector(collectKey);
        //获取远程 URL 栏目网页代码（新闻列表）(缓存？5分钟)
        Ue.Qcdz.Domain.HttpRequest http = new Ue.Qcdz.Domain.HttpRequest();
        if (!string.IsNullOrEmpty(collModel.Encode))
            http.Encode = collModel.Encode;
        string rHtmlCode = http.Get(collModel.SiteUrl);
        if (string.IsNullOrEmpty(rHtmlCode))
            return ObjectToJson(null, 0, "ERRE:采集:" + collModel.SiteUrl + " 到得内容为空!");
        //获取栏目HtmlCode
        string columnHtmlCode = GetRegexHtmlCode(collModel.Outskirts, rHtmlCode);
        if (string.IsNullOrEmpty(columnHtmlCode))
            return ObjectToJson(null, 0, "ERRE:没有找到该栏目");
        //获取栏目的A链接列表
        int rCount;
        string aLink = GetRegexHtmlCode(collModel.ALink, columnHtmlCode, rIndex, out rCount);
        if (rCount == 0)
            return ObjectToJson(null, rCount, "ERRE:没有找到匹配的内容链接！");
        //本次采集完成
        if (rIndex > rCount)
        {
            //更新用户最新操作（待完成）

            //返回
            return ObjectToJson(null, rCount, "complete");
        }
        //if (totalCount > collectCount)
        //{
        //    return ObjectToJson(null, rCount, "complete");
        //}

        //获取单条新闻HTMLCODE
        //获取 href 属性
        aLink = GetLinkHrefAttr(aLink);
        //获取完整路径
        aLink = GetFullLink(aLink, collModel.SiteUrl);
        string newsHtmlCode = http.GetHTML(aLink);
        if (string.IsNullOrEmpty(newsHtmlCode))
            return ObjectToJson(null, rCount, "采集此链接" + aLink + "得到的内容为空！");
        //获取文章的BOX
        string newsBoxHtmlCode = GetRegexHtmlCode(collModel.NewsOutskirts, newsHtmlCode);
        if (string.IsNullOrEmpty(newsBoxHtmlCode))
            return ObjectToJson(null, rCount, "提示:该Url不符合规则，放弃采集:" + aLink);
      
        //分类：
        Ue.Qcdz.Domain.Tags.Tag tag = null;
        if (!string.IsNullOrEmpty(nodeKey))
        {
            Ue.Qcdz.Domain.TreeViews.TreeNode node = ArticleService.TagTreeView.FindNodeBy(nodeKey);
            tag = Ue.Qcdz.Domain.Tags.TagService.GetTag(node.Tag.Key);
        }
        Article model = new Article(Guid.NewGuid(), tag);
        //获取新闻标题
        model.Title = DealHtml(GetRegexHtmlCode(collModel.Title, newsBoxHtmlCode, "Title"));
        model.Title = CutLen(model.Title, 200);
        //获取作者
        model.Editor = DealHtml(GetRegexHtmlCode(collModel.Editor, newsBoxHtmlCode, "Editor"));
        model.Editor = CutLen(model.Editor, 200);
        //获取来源
        model.Reference = DealHtml(GetRegexHtmlCode(collModel.Reference, newsBoxHtmlCode, "Reference"));
        model.Reference = CutLen(model.Reference, 200);
        //图片URL
        string imageUrl = DealHtml(GetRegexHtmlCode(collModel.ImageUrl, newsBoxHtmlCode, "ImageUrl"));
        model.ImageUrl = GetFullLink(imageUrl, collModel.SiteUrl);
        model.ImageUrl = CutLen(model.ImageUrl, 200);
        //获取新闻内容
        model.Content = GetRegexHtmlCode(collModel.Content, newsBoxHtmlCode, "Content");
        //保存数据入库
        if (collModel.Status == 1)
        {
            //不要重复写入
            Article dbArticle = Ue.Qcdz.Domain.Articles.ArticleService.GetArticle(model.Title);
            if (dbArticle != null)
                return ObjectToJson(model, rCount, "采集成功，该文章已经存在，不重复写入！：" + aLink);
            Ue.Qcdz.Domain.Articles.ArticleService.AddArticle(model);
            totalCount++;
            return ObjectToJson(model, rCount, "ok，采集成功,数据已保存。" + aLink);
        }
        else
        {
            //测试采集成功！
            return ObjectToJson(model, rCount, "ok，测试采集成功,没有保存当前采集数据！," + aLink);
        }
    }
    #region ：私有方法
    private string ObjectToJson(Article model, int rCount, string msgStatus)
    {
        if (model == null)
            model = new Article(Guid.NewGuid());
        StringBuilder json = new StringBuilder();
        json.Append("[{");
        json.Append("\"status\":\"" + msgStatus + "\",");
        json.Append("\"rCount\":\"" + rCount + "\",");
        json.Append("\"Title\":\"" + Server.HtmlEncode(model.Title) + "\",");
        json.Append("\"Editor\":\"" + Server.HtmlEncode(model.Editor) + "\",");
        json.Append("\"Reference\":\"" + Server.HtmlEncode(model.Reference) + "\",");
        json.Append("\"ImageUrl\":\"" + Server.HtmlEncode(model.ImageUrl) + "\",");
        json.Append("\"Content\":\"内容长度：" + GetLen(model.Content) + "\"");
        json.Append("}]");
        return json.ToString();
    }

    private string GetLen(string str)
    {
        if (!string.IsNullOrEmpty(str) && str.Length > 10)
            return str.Length.ToString();
        return str;
    }
    /// <summary>
    /// 
    /// </summary>
    private string CutLen(string str, int len)
    {
        if (!string.IsNullOrEmpty(str) && str.Length > len)
            return str.Substring(0, len);
        return str;
    }

    /// <summary>
    /// 
    /// </summary>
    /// <param name="regString">正则表达式</param>
    /// <param name="htmlCode">源码</param>
    /// <param name="rGroupName">返回此组的内容</param>
    /// <returns></returns>
    private string GetRegexHtmlCode(string regString, string htmlCode, string rGroupName)
    {
        if (string.IsNullOrEmpty(regString) || string.IsNullOrEmpty(htmlCode) || string.IsNullOrEmpty(rGroupName))
            return string.Empty;

        Regex reg = new Regex(regString, RegexOptions.Singleline);
        MatchCollection mat = reg.Matches(htmlCode);
        if (mat.Count > 0)
        {
            foreach (Match m in mat)
            {
                if (m.Groups[rGroupName] != null && !string.IsNullOrEmpty(m.Groups[rGroupName].Value))
                    return m.Groups[rGroupName].Value;
                else
                    return m.Groups[0].Value;
            }
        }
        return string.Empty;
    }

    /// <summary>
    /// 获取链接的的 Href 属性
    /// </summary>
    /// <param name="aLink"></param>
    /// <returns></returns>
    private string GetLinkHrefAttr(string aLink)
    {
        string hrefReg = "href\\s*=\\s*(?<href>([\"\'])?(?(1)(.*?)\\1|([^\\s\\>]+)))[^>]";
        return GetRegexHtmlCode(hrefReg, aLink, "href");
    }
    /// <summary>
    /// 
    /// </summary>
    /// <param name="regString"></param>
    /// <param name="htmlCode"></param>
    /// <returns></returns>
    private string GetRegexHtmlCode(string regString, string htmlCode)
    {
        if (string.IsNullOrEmpty(regString) || string.IsNullOrEmpty(htmlCode))
            return string.Empty;

        RegexOptions options = RegexOptions.Singleline | RegexOptions.Multiline | RegexOptions.IgnoreCase;

        Regex reg = new Regex(regString, options);
        MatchCollection mat = reg.Matches(htmlCode);

        if (mat.Count > 0)
        {
            foreach (Match m in mat)
            {
                return m.Groups[0].Value;
            }
        }
        return string.Empty;
    }
    private string GetRegexHtmlCode(string regString, string htmlCode, int rIndex, out int rCount)
    {
        RegexOptions options = RegexOptions.Singleline | RegexOptions.Multiline | RegexOptions.IgnoreCase;
        Regex reg = new Regex(regString, options);
        MatchCollection mat = reg.Matches(htmlCode);
        rCount = mat.Count;
        if (mat.Count == 0)
            return string.Empty;
        //已经达到最大的链接数目
        if (rIndex >= mat.Count)
            return string.Empty;
        return mat[rIndex].Groups[0].Value;
    }
    private string GetFullLink(string link, string siteUrl)
    {
        link = link.Replace("href=", "");
        link = link.Replace("\"", "");
        link = link.Replace("'", "");
        link = link.Trim();
        //获取文章URLDE 内容
        if (string.IsNullOrEmpty(link))
            return string.Empty;
        //如果不是绝对路径
        if (link.IndexOf("http") != 0)
        {
            System.Uri url = new Uri(siteUrl);
            link = url.Host + "/" + link;
            link = link.Replace("//", "/");
            link = "http://" + link;
        }
        return link;
    }
    /// <summary>
    /// 去掉文章内容的 标签
    /// </summary>
    private static string DealHtml(string str)
    {
        str = Regex.Replace(str, @"\<(img)[^>]*>|<\/(img)>", "", RegexOptions.IgnoreCase);
        str = Regex.Replace(str, @"\<(table|tbody|tr|td|th|)[^>]*>|<\/(table|tbody|tr|td|th|)>", "", RegexOptions.IgnoreCase);
        str = Regex.Replace(str, @"\<(div|blockquote|fieldset|legend)[^>]*>|<\/(div|blockquote|fieldset|legend)>", "", RegexOptions.IgnoreCase);
        str = Regex.Replace(str, @"\<(font|i|u|h[1-9]|s)[^>]*>|<\/(font|i|u|h[1-9]|s)>", "", RegexOptions.IgnoreCase);
        str = Regex.Replace(str, @"\<(style|strong)[^>]*>|<\/(style|strong)>", "", RegexOptions.IgnoreCase);
        str = Regex.Replace(str, @"\<a[^>]*>|<\/a>", "", RegexOptions.IgnoreCase);
        str = Regex.Replace(str, @"\<(meta|iframe|frame|span|tbody|layer)[^>]*>|<\/(iframe|frame|meta|span|tbody|layer)>", "", RegexOptions.IgnoreCase);
        str = Regex.Replace(str, @"\<br[^>]*", "", RegexOptions.IgnoreCase);
        return str;
    }
    #endregion
</script>
